# aggregate all the domain stat

require "dbm"
require "00_common"



module DomainAggre
  def self.get_all_domain_stat_files
    Common.get_all_files "./stat_domain"
  end

  def self.site_counter(domain, value, key_words, all_mapping)
    if domain.include? key_words
      stat_key = "==" + key_words
      all_mapping[stat_key] = all_mapping[stat_key] ? (all_mapping[stat_key] + value ) : value
    end
  end

  def self.aggregate_all_files(all_files)
    all_mapping = Hash.new
    all_mapping["total"] = 0

    
    all_files.each do |file_name|
      one_mapping = get_one_mapping(file_name)
      one_mapping.each do |key, value|
        domain = Common.extract_domain(key)

        all_mapping["total"] = all_mapping["total"] + value

        # puts domain
        if all_mapping[domain]
          all_mapping[domain] = all_mapping[domain] + value
        else
          all_mapping[domain] = value
        end


        #
        # for social networking
        #
        DomainAggre.site_counter domain, value, "facebook", all_mapping
        DomainAggre.site_counter domain, value, "fbcdn", all_mapping
        DomainAggre.site_counter domain, value, "twitter", all_mapping
        DomainAggre.site_counter domain, value, "myspace", all_mapping
        DomainAggre.site_counter domain, value, "hi5", all_mapping
        DomainAggre.site_counter domain, value, "ning", all_mapping
        DomainAggre.site_counter domain, value, "orkut", all_mapping
        DomainAggre.site_counter domain, value, "friendster", all_mapping
        DomainAggre.site_counter domain, value, "zynga", all_mapping

        #
        # for search engine
        #        
        DomainAggre.site_counter domain, value, "google", all_mapping
        DomainAggre.site_counter domain, value, "bing", all_mapping
        DomainAggre.site_counter domain, value, "yahoo", all_mapping
        DomainAggre.site_counter domain, value, "baidu", all_mapping
        DomainAggre.site_counter domain, value, "youtube", all_mapping
        DomainAggre.site_counter domain, value, "blogspot", all_mapping

      end
    end

    all_mapping
  end

  def self.get_one_mapping(file_name)
    file = File.open "./stat_domain/" + file_name, "r"
    one_mapping = Hash.new
    begin
      while line = file.readline
        fields = line.split "|"
        key = fields[0]
        value = fields[1].to_i

        one_mapping[key] = value
      end
    rescue Exception => e
      # puts e.to_s
      file.close
    end

    one_mapping
  end

  def self.output_all_mapping(all_mapping)
    output_file = File.open "domain_stat_all_3.csv", "w"
    all_mapping.each do |key,value|  
      output_file.puts "#{key},#{value}"
    end
    output_file.close
  end

  def self.top_100(all_mapping)
    output_file = File.open "domain_top_100.csv", "w"
    top_mapping = Hash.new

    #
    # search for top domains
    #
    all_mapping.each do |domain,num|  
      if top_mapping.size < 100
        top_mapping[domain] = num
      else
        min_num = 100000000
        min_domain = ""
        top_mapping.each do |top_domain,top_num|
          if top_num < min_num
            min_domain = top_domain
            min_num = top_num
          end
        end

        if num > min_num
          top_mapping.delete min_domain
          top_mapping[domain] = num
        end
      end
    end

    #
    # output topdomains
    #
    # TODO
    top_mapping.each do |key, value|
      output_file.puts "#{key},#{value}"
    end
    output_file.close
  end
end

begin
  all_files = DomainAggre.get_all_domain_stat_files
  puts all_files.join "\n"

  puts "---------"

  all_mapping = DomainAggre.aggregate_all_files all_files
  DomainAggre.output_all_mapping all_mapping
  DomainAggre.top_100 all_mapping

rescue Exception => e
  puts "ERROR:" + e.to_s 
end